import os 
import re
import sys 
import shelve 

BASE_DIR =  'wiki_textdump'

WC_STATS = shelve.open('wcstat.db')
G_WC_STATS = shelve.open('g_wcstats.db')
REV_IDX = shelve.open('r_idx.db')

MAX_RECS = 6000

stoplist = [] 

def cleanText(someText):
    someTextP = re.compile('\W')
    someText = someTextP.sub(" ",someText)
    return someText 

def updateWCStats(someHash,someText):
    someText = someText.strip()
    if(someText!="" and len(someText.split()) >1):
        for eachToken in someText.split():
            if(eachToken in stoplist):
                continue
            
            if(someHash.has_key(eachToken) == False):
                someHash[eachToken] = 0

            someHash[eachToken] += 1         
    return someHash

def genWC(fileName):
    
    fileToRead = open(fileName)
    p = re.compile('\[.+\]')
    wc = {} 
    for eachLine in fileToRead:
        eachLine = p.sub('',eachLine)
        eachLine = eachLine.strip()

        if(eachLine == "External links" or eachLine == "References"):
            break
        wc = updateWCStats(wc,cleanText(eachLine.lower()))
    return wc 


def getArtName(fileName):
    fileName = os.path.join(BASE_DIR,eachFile)
    fileName = fileName.replace(".html","")
    result = ""
    
    if(len(fileName.split("_")) > 5):
        if(len(fileName.split("_")[5:]) == 1):
            curr_file_name = ''.join(fileName.split("_")[5:])
            result = curr_file_name
        else:
            tmp = fileName.split("_")[len(fileName.split("_"))-1]
            tmp = tmp.strip()

            if(len(tmp) != 4):
                result = '_'.join(fileName.split("_")[5:len(fileName.split("_"))])
            else:
                result = '_'.join(fileName.split("_")[5:len(fileName.split("_"))-1])

    if(result != "" and result[0] == "_"):
        result = result[1:]

    return result 

def init_stoplist():
    fileToRead = open('stoplist.txt')

    for eachLine in fileToRead:
        eachLine = eachLine.strip()
        stoplist.append(eachLine)

def genHashStr(someHash):
    result = ""
    for eachKey in someHash.keys():
        result += eachKey+":"+str(someHash[eachKey])+"\t"
    return result
    
if __name__ == "__main__":
    init_stoplist()
    rec_count = 0
    last_check_rec_count = -1
    file_count = 0 

    for eachFile in os.listdir(BASE_DIR):
        if(rec_count > MAX_RECS and (rec_count % MAX_RECS == 0) and (rec_count != last_check_rec_count)):

            WC_STATS.close()
            G_WC_STATS.close()
            REV_IDX.close()

            file_count += 1 

            WC_STATS = shelve.open('wcstat'+str(file_count)+'.db')
            G_WC_STATS = shelve.open('g_wcstats'+str(file_count)+'.db')
            REV_IDX = shelve.open('r_idx'+str(file_count)+'.db')

            print "Processed:",str(rec_count)
            last_check_rec_count = rec_count
        
        art_name = getArtName(eachFile)

        eachFile = os.path.join(BASE_DIR,eachFile)        
        wc = genWC(eachFile)
        
        #gen wc stats
        WC_STATS[art_name] = genHashStr(wc)
        
        #gen global stats
        for eachWord in wc.keys():
            if(G_WC_STATS.has_key(eachWord) == False):
                G_WC_STATS[eachWord] = ""

            G_WC_STATS[eachWord] += "\t" + str(wc[eachWord])
        
        #gen rev idx
        for eachWord in wc.keys():

            if(REV_IDX.has_key(eachWord) == False):
                REV_IDX[eachWord] = "" 

            REV_IDX[eachWord] += "\t" + art_name

        rec_count += 1
        

